# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import time
import re

from pymongo import MongoClient


class SmzdmPipeline(object):
    def open_spider(self, spider):
        # 连接数据库
        client = MongoClient('127.0.0.1', 27017)
        self.collection = client['smzdm']['youhui']

    def process_item(self, item, spider):
        # 获取数据库数据的数量
        if self.collection.find().count() >= 1000:
            print('数据已达1000条,请及时停止程序')
            return item
        # 价格使用正则进行格式化
        price = []
        if re.search('(\d+(\.\d+)?)元', item['price']):
            price.append(re.search('(\d+(\.\d+)?)元', item['price']).group(1))
        if re.search('包邮', item['price']):
            price.append(re.search('包邮', item['price']).group())
        if re.search('（.*）', item['price']):
            price.append(re.search('（.*）', item['price']).group()[1:-1])
        item['price'] = price
        # 日期格式化
        if item['date'] != None and re.search(':', item['date']):
            date = time.strftime('%Y-%m-%d', time.localtime(time.time()))
        elif item['date'] != None:
            date = time.strftime('%Y-', time.localtime(time.time())) + str(item['date'])
        else:
            date = None
        item['date'] = date
        # 判断是否存在以判断是否存贮数据
        if self.collection.find_one(item):
            print('数据已存在')
        else:
            print('存贮数据...', item)
            self.collection.insert_one(item)


        return item